{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "from PIL import Image\n",
    "from PIL import ImageDraw\n",
    "import math\n",
    "plt.style.use({'figure.figsize':(10, 10)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "Epsilon_start=1\n",
    "Epsilon_final=0.01\n",
    "Robot_radium=40\n",
    "Decay_rate=0.000001#he dacaying rate of the Epsilon, the range of the epsilon is 0.01-1, initially it is 1.\n",
    "Action_times=0 #Rigister the totality of the times of selecting actions, including the random selections and selection based on Q_Table\n",
    "Velocity_tripod=0.289*Robot_radium\n",
    "Beta=0.9\n",
    "Alpha=0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Up</th>\n",
       "      <th>Left_45D</th>\n",
       "      <th>Right_45D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>L60D0/30</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>L180D0/30</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>L250D0/30</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>L300D0/30</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>L60D30/60</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Up  Left_45D  Right_45D\n",
       "L60D0/30   0.0       0.0        0.0\n",
       "L180D0/30  0.0       0.0        0.0\n",
       "L250D0/30  0.0       0.0        0.0\n",
       "L300D0/30  0.0       0.0        0.0\n",
       "L60D30/60  0.0       0.0        0.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Q_table2_states=np.array(['L60D0/30','L180D0/30','L250D0/30','L300D0/30',\n",
    "                'L60D30/60','L180D30/60','L250D30/60','L300D30/60',\n",
    "                 'L60D60/90','L180D60/90','L250D60/90','L300D60/90',\n",
    "                'L60D90/120','L180D90/120','L250D90/120','L300D90/120',\n",
    "                'L60D120/150','L180D120/150','L250D120/150','L300D120/150',\n",
    "                'L60D150/180','L180D150/180','L250D150/180','L300D150/180',\n",
    "                'L60D180/210','L180D180/210','L250D180/210','L300D180/210',\n",
    "                'L60D210/240','L180D210/240','L250D210/240','L300D210/240',\n",
    "                'L60D240/270','L180D240/270','L250D240/270','L300D240/270',\n",
    "                'L60D270/300','L180D270/300','L250D270/300','L300D270/300',\n",
    "                'L60D300/330','L180D300/330','L250D300/330','L300D300/330',\n",
    "                 'L60D330/360','L180D330/360','L250D330/360','L300D330/360'])\n",
    "Q_table2_action=np.array(['Up','Left_45D','Right_45D'])\n",
    "Q_table2_actions_length=len(Q_table2_action)\n",
    "Q_table2_states_length=len(Q_table2_states)\n",
    "Q_table2_real=np.zeros((Q_table2_states_length,Q_table2_actions_length))\n",
    "Q_table2_real=pd.DataFrame(Q_table2_real,columns=Q_table2_action,index=Q_table2_states)\n",
    "Q_table2_real.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Set_destination(x,y):\n",
    "    return x,y\n",
    "# Destination_x,Destination_y=Set_destination(900,900)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlsAAAJCCAYAAAD3HAIiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFkBJREFUeJzt3V+oZed53/HfU00s/2sSicAgz6j1XEwSxoXgZFCduIRS\np0htQkZXQgG309RUF1VSJxQSKb3orS9CSAq1YbCdTImREIpBQ2j+KEog7UUkj6WCLSmqBquyRhlJ\nLiZxCEW25KcXe8lsyx7NzDnnOWefmc8HhrP2u9da+z28aPSd/W9VdwcAgBl/b68nAABwNRNbAACD\nxBYAwCCxBQAwSGwBAAwSWwAAg3Y9tqrqtqp6pqrOVdU9u/34AAC7qXbze7aq6rok/zvJP09yPsnn\nkvxcdz+1a5MAANhFu/3M1i1JznX3l7r760nuT3Jil+cAALBrDuzy4x1K8sLa7fNJ/vGbd6qqu5Lc\ntdz8sV2YFwDAt3R37dS5dju2Lkt3n0pyKkmqyvWEAIB9a7dfRnwxyc1rtw8vYwAAV6Xdjq3PJTla\nVUeq6m1J7kxyZpfnAACwa3b1ZcTufq2qfiHJHyW5Lsmnu/vJ3ZwDAMBu2tWvftgK79kCAHbbTr5B\n3jfIAwAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADDowF5PYK90915PAQDYAVW111N4S57ZAgAYJLYAAAaJLQCAQWIL\nAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWIL\nAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWIL\nAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWIL\nAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWIL\nAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWIL\nAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWIL\nAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBW46tqrq5qv6sqp6qqier6qPL+I1V\n9XBVPbv8vGHtmHur6lxVPVNVt+7ELwAAsMmqu7d2YNVNSW7q7ser6u8n+XyS25P8myRf7e6PVdU9\nSW7o7l+tqmNJ7ktyS5L3JPmTJD/Y3a9f4nG2NsFL2OrvDQBslqra8XN2946ddMvPbHX3he5+fNn+\n2yRPJzmU5ESS08tup7MKsCzj93f3q939XJJzWYUXAMBVa0fes1VV703y/iSPJjnY3ReWu15KcnDZ\nPpTkhbXDzi9j3+18d1XV2ao6uxPzAwDYK9uOrap6d5LfS/JL3f219ft69VrdFb9e192nuvt4dx/f\n7vwAAPbStmKrqr4nq9D6THd/dhl+eXk/1xvv63plGX8xyc1rhx9exgAArlrb+TRiJflUkqe7+zfW\n7jqT5OSyfTLJQ2vjd1bV9VV1JMnRJI9t9fEBAPaD7Xwa8Z8k+R9JvpDkm8vwr2X1vq0HkvyDJM8n\nuaO7v7oc85+S/Nskr2X1suMfXMbj+DQiAHBRm/5pxC3H1m4RWwDAW9n02PIN8gAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QWAMAgsQUA\nMEhsAQAM2nZsVdV1VfVEVf3+cvvGqnq4qp5dft6wtu+9VXWuqp6pqlu3+9gAAJtuJ57Z+miSp9du\n35Pkke4+muSR5Xaq6liSO5O8L8ltST5eVdftwOMDAGysbcVWVR1O8tNJPrk2fCLJ6WX7dJLb18bv\n7+5Xu/u5JOeS3LKdxwcA2HTbfWbrN5P8SpJvro0d7O4Ly/ZLSQ4u24eSvLC23/ll7DtU1V1Vdbaq\nzm5zfgAAe2rLsVVVP5Pkle7+/MX26e5O0ld67u4+1d3Hu/v4VucHALAJDmzj2A8m+dmq+pdJ3p7k\ne6vqd5O8XFU3dfeFqropySvL/i8muXnt+MPLGADAVWvLz2x1973dfbi735vVG9//tLs/nORMkpPL\nbieTPLRsn0lyZ1VdX1VHkhxN8tiWZw4AsA9s55mti/lYkgeq6iNJnk9yR5J095NV9UCSp5K8luTu\n7n594PEBADZGrd5WtbmqamSCm/57AwCXp6p2/JzdvWMn9Q3yAACDxBYAwCCxBQAwSGwBAAwSWwAA\ng8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAA\ng8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAA\ng8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAA\ng8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAA\ng8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAA\ng8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAA\ng8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAA\ng8QWAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwaFuxVVXfX1UP\nVtVfVtXTVfXjVXVjVT1cVc8uP29Y2//eqjpXVc9U1a3bnz4AwGbb7jNbv5XkD7v7h5P8SJKnk9yT\n5JHuPprkkeV2qupYkjuTvC/JbUk+XlXXbfPxAQA22pZjq6q+L8lPJvlUknT317v7r5OcSHJ62e10\nktuX7RNJ7u/uV7v7uSTnktyy1ccHANgPtvPM1pEkX0ny21X1RFV9sqreleRgd19Y9nkpycFl+1CS\nF9aOP7+MAQBctbYTWweS/GiST3T3+5P8XZaXDN/Q3Z2kr/TEVXVXVZ2tqrPbmB8AwJ7bTmydT3K+\nux9dbj+YVXy9XFU3Jcny85Xl/heT3Lx2/OFl7Dt096nuPt7dx7cxPwCAPbfl2Orul5K8UFU/tAx9\nKMlTSc4kObmMnUzy0LJ9JsmdVXV9VR1JcjTJY1t9fACA/eDANo//xSSfqaq3JflSkp/PKuAeqKqP\nJHk+yR1J0t1PVtUDWQXZa0nu7u7Xt/n4AAAbrVZvq9pcVTUywU3/vQGAy1NVO37O7t6xk/oGeQCA\nQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCA\nQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCA\nQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCA\nQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCA\nQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCA\nQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCA\nQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCA\nQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABoktAIBBYgsAYJDYAgAYJLYAAAaJLQCA\nQWILAGCQ2AIAGCS2AAAGbSu2quqXq+rJqvpiVd1XVW+vqhur6uGqenb5ecPa/vdW1bmqeqaqbt3+\n9AEANlt199YOrDqU5H8mOdbd/6+qHkjy35McS/LV7v5YVd2T5Ibu/tWqOpbkviS3JHlPkj9J8oPd\n/folHmdrE7yErf7eAMBmqaodP2d379hJt/sy4oEk76iqA0nemeSvkpxIcnq5/3SS25ftE0nu7+5X\nu/u5JOeyCi8AgKvWlmOru19M8utJvpzkQpK/6e4/TnKwuy8su72U5OCyfSjJC2unOL+MfYeququq\nzlbV2a3ODwBgE2w5tpb3Yp1IciSrlwXfVVUfXt+nV6/VXfHrdd19qruPd/fxrc4PAGATbOdlxJ9K\n8lx3f6W7v5Hks0l+IsnLVXVTkiw/X1n2fzHJzWvHH17GAACuWtuJrS8n+UBVvbNW70z7UJKnk5xJ\ncnLZ52SSh5btM0nurKrrq+pIkqNJHtvG4wMAbLwDWz2wux+tqgeTPJ7ktSRPJDmV5N1JHqiqjyR5\nPskdy/5PLp9YfGrZ/+5LfRIRAGC/2/JXP+wWX/0AALyVq/2rHwAAeAtiCwBgkNgCABgktgAABokt\nAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABokt\nAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABokt\nAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBg0IG9ngAAsI9UvfX9\n3bszj31EbAEAl3apyHrzfqLrW8QWAHBxlxtZFztOdHnPFgBwEVsNLb6N2AIA5gg2sQUAfBciaceI\nLQBg1jUebmILAPh213gc7TSxBQDMu4YDTmwBAAwSWwAAg8QWAMAgsQUAMEhsAQDzruHL9ogtAIBB\nYgsA+HbX8LNQE8QWADDrGo83sQUAfKdrPJB2ktgCAOaINrEFAFxE9/ZiSWglSQ7s9QQAgA33RjRd\n7vUNRda3EVsAwOV5q+gSWBcltgCAKyOsroj3bAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QW\nAMAgsQUAMEhsAQAMElsAAIPEFgDAILEFADBIbAEADBJbAACDxBYAwCCxBQAwSGwBAAwSWwAAg8QW\nAMAgsQUAMEhsAQAMOrDXE9grVbXXUwAArgGe2QIAGCS2AAAGiS0AgEFiCwBgkNgCABgktgAABokt\nAIBBYgsAYJDYAgAYJLYAAAaJLQCAQWILAGCQ2AIAGCS2AAAGiS0AgEFiCwBgkNgCABh0ydiqqk9X\n1StV9cW1sRur6uGqenb5ecPaffdW1bmqeqaqbl0b/7Gq+sJy33+pqtr5XwcAYLNczjNbv5PktjeN\n3ZPkke4+muSR5Xaq6liSO5O8bznm41V13XLMJ5L8uyRHlz9vPicAwFXnkrHV3X+e5KtvGj6R5PSy\nfTrJ7Wvj93f3q939XJJzSW6pqpuSfG93/0V3d5L/tnYMAMBV68AWjzvY3ReW7ZeSHFy2DyX5i7X9\nzi9j31i23zz+XVXVXUnuWm6+muSLF9uXjfQDSf7vXk+Cy2a99h9rtv9Ys/3lH1bVXd19aidOttXY\n+pbu7qrqnZjM2jlPJTmVJFV1truP7+T5mWXN9hfrtf9Ys/3Hmu0/VXU2S4ts11Y/jfjy8tJglp+v\nLOMvJrl5bb/Dy9iLy/abxwEArmpbja0zSU4u2yeTPLQ2fmdVXV9VR7J6I/xjy0uOX6uqDyyfQvzX\na8cAAFy1LvkyYlXdl+SfJvmBqjqf5D8n+ViSB6rqI0meT3JHknT3k1X1QJKnkryW5O7ufn051b/P\n6pON70jyB8ufy7EjT+Gxq6zZ/mK99h9rtv9Ys/1nx9asVh8OBABggm+QBwAYJLYAAAZtbGxV1W3L\nJX/OVdU9ez0fVqrq5qr6s6p6qqqerKqPLuNXfAkndk9VXVdVT1TV7y+3rdcGq6rvr6oHq+ovq+rp\nqvpxa7bZquqXl78Tv1hV91XV263ZZtnLyw9uZGwtl/j5r0n+RZJjSX5uuRQQe++1JP+xu48l+UCS\nu5e12colnNg9H03y9Npt67XZfivJH3b3Dyf5kazWzpptqKo6lOQ/JDne3f8oyXVZrYk12yy/kz26\n/OBGxlaSW5Kc6+4vdffXk9yf1aWA2GPdfaG7H1+2/zar/wkcyhVewml3Z31tq6rDSX46ySfXhq3X\nhqqq70vyk0k+lSTd/fXu/utYs013IMk7qupAkncm+atYs42yl5cf3NTYOpTkhbXbb3l5H/ZGVb03\nyfuTPJq3voSTtdxbv5nkV5J8c23Mem2uI0m+kuS3l5d+P1lV74o121jd/WKSX0/y5SQXkvxNd/9x\nrNl+cKVrdChXcPnBN2xqbLHhqurdSX4vyS9199fW71tq33eKbICq+pkkr3T35y+2j/XaOAeS/GiS\nT3T3+5P8XZaXNt5gzTbL8j6fE1mF8nuSvKuqPry+jzXbfJNrtKmxdbHL/rABqup7sgqtz3T3Z5fh\nK72EE7vjg0l+tqr+T1Yvx/+zqvrdWK9Ndj7J+e5+dLn9YFbxZc02108lea67v9Ld30jy2SQ/EWu2\nH+zK5Qc3NbY+l+RoVR2pqrdl9Sa1M3s8J5Isn7r4VJKnu/s31u66oks47dZ8r3XdfW93H+7u92b1\n39GfdveHY702Vne/lOSFqvqhZehDWV2Vw5ptri8n+UBVvXP5O/JDWb2f1Zptvl25/OAlL9ezF7r7\ntar6hSR/lNWnOj7d3U/u8bRY+WCSf5XkC1X1v5axX8vWLuHE3rFem+0Xk3xm+cfml5L8fFb/OLZm\nG6i7H62qB5M8ntUaPJHVpV7eHWu2MWoPLz/ocj0AAIM29WVEAICrgtgCABgktgAABoktAIBBYgsA\nYJDYAgAYJLYAAAb9f3n35cMwEAlBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c248277358>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Draw_map2 funcion is used to train the first Q-Table2, which allows it to arrive to the destination\n",
    "def Draw_map(Destination_x,Destination_y):\n",
    "    im = Image.new(\"RGB\", size=(1000,1000),color=(0,0,0)) \n",
    "    draw = ImageDraw.Draw(im,mode='RGB')\n",
    "    draw.rectangle((40,40,960,960),(255,255,255),(255,255,255))\n",
    "    draw.ellipse((Destination_x-20,Destination_y-20,Destination_x+20,Destination_y+20), (255,0,0), (255,0,0)) \n",
    "    return im\n",
    "# im=Draw_map(900,900)\n",
    "# plt.imshow(im)\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Initial_Q_Table2(LengthOfActions,LengthOfStates):\n",
    "    Q_table2=np.zeros((LengthOfStates,LengthOfActions))\n",
    "    print('***********************************************************')\n",
    "    print(\"Succeed to initialize Q-table2!\")\n",
    "    print('***********************************************************')\n",
    "    return Q_table2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Robot_destination(Destination_x,Destination_y,Current_x,Current_y,Current_angle):\n",
    "    #Left-negative,180-360 |||| Right-positive,0-180\n",
    "    Delta_x=Destination_x-Current_x\n",
    "    Delta_y=Destination_y-Current_y\n",
    "    Distance=np.sqrt(Delta_x*Delta_x+Delta_y*Delta_y)\n",
    "    Direct_angle=(np.arctan(Delta_y/Delta_x))*180/np.pi\n",
    "    if Delta_x<0:\n",
    "        Direct_angle=Direct_angle+180\n",
    "    True_angle=(Current_angle-Direct_angle)%360\n",
    "#     print(true_angle)\n",
    "    if True_angle>180:\n",
    "        return 360-True_angle,'right',Distance\n",
    "    else:\n",
    "        return True_angle,'left',Distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Is_Crash function, in the second map, is used to judge if the robot collides with the black frame\n",
    "def Is_Crash(Current_x,Current_y,im):\n",
    "    Crash=False\n",
    "    Degree=[-150,-120,-90,-60,-30,0,30,60,90,120,150,180]\n",
    "    Dis=np.arange(0,50,5)\n",
    "    for i in Dis:\n",
    "        for j in Degree:\n",
    "            x=Current_x+i*np.cos(j/180*np.pi)\n",
    "            y=Current_y+i*np.sin(j/180*np.pi)\n",
    "            if (im.getpixel((x,y)))==(0,0,0):\n",
    "                Crash=True\n",
    "                break\n",
    "        if Crash==True:\n",
    "                break\n",
    "    return Crash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Is_arrive(Current_x,Current_y,im):\n",
    "    Arrive=False\n",
    "    Crash=False\n",
    "    Degree=[-150,-120,-90,-60,-30,0,30,60,90,120,150,180]\n",
    "    Distance=np.arange(0,20,5)\n",
    "    for i in Distance:\n",
    "        for j in Degree:\n",
    "            x=Current_x+i*np.cos(j/180*np.pi)\n",
    "            y=Current_y+i*np.sin(j/180*np.pi)\n",
    "            if (im.getpixel((x,y)))==(0,0,0):\n",
    "                Crash=True\n",
    "                break\n",
    "            if (im.getpixel((x,y)))==(255,0,0):\n",
    "                Arrive=True\n",
    "                break\n",
    "        if Arrive==True or Crash==True:\n",
    "                break\n",
    "    return Arrive\n",
    "# print(Is_arrive(500,500))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Random_start function, in the second map, is used to set the beginning point in the white map.\n",
    "def Random_start(im):\n",
    "    Angle=0\n",
    "    x,y=np.random.random(2)*1000\n",
    "    while(Is_Crash(x,y,im)==True or Is_arrive(x,y,im)==True):\n",
    "#         print('Boom')\n",
    "        x,y=np.random.random(2)*1000\n",
    "    return x,y,Angle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Output_state_index(True_angle,Left_right,Distance):\n",
    "    if 0<=Distance<60:\n",
    "        Distance_level=0\n",
    "    elif 60<=Distance<180:\n",
    "        Distance_level=1\n",
    "    elif 180<=Distance<250:\n",
    "        Distance_level=2\n",
    "    else:\n",
    "        Distance_level=3\n",
    "    Angle_level=True_angle//30\n",
    "    \n",
    "    if Left_right=='right':\n",
    "        Left_right_level=1\n",
    "        State_number=int(Distance_level+24*Left_right_level+(5-Angle_level)*4)\n",
    "    elif Left_right=='left':\n",
    "        Left_right_level=0\n",
    "        Angle_level=True_angle//30\n",
    "        State_number=int(Distance_level+24*Left_right_level+Angle_level*4)\n",
    "    return State_number,Distance_level,Angle_level\n",
    "# State_Number,_,_=Output_state_index(20,'right',299) #180 belongs to 180-210,''right' does not have 180\n",
    "# Q_table2_states[State_Number]\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Choose_action2(Q_Table,Current_state,Action_times):\n",
    "    Epsilon=Epsilon_final+(Epsilon_start-Epsilon_final)*np.exp(-1*Decay_rate*Action_times)\n",
    "    State_action=Q_Table[Current_state,:]\n",
    "    if(np.random.random()<Epsilon or np.all(State_action==[0])):\n",
    "        Next_action=np.random.randint(Q_table2_actions_length)\n",
    "    else:\n",
    "        Next_action=np.argmax(State_action)\n",
    "    return Next_action\n",
    "# print(Choose_action2(Q_Table2,20,0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Output_next_state(Current_x,Current_y,Current_angle,Destination_x,Destination_y,Action,im):\n",
    "    Arrive=False\n",
    "    Crash=False\n",
    "    if Action==0:\n",
    "        Next_x=Current_x+Velocity_tripod*np.cos(Current_angle/180*np.pi)\n",
    "        Next_y=Current_y+Velocity_tripod*np.sin(Current_angle/180*np.pi)\n",
    "        Next_angle=Current_angle\n",
    "        if Is_Crash(Next_x,Next_y,im)==True:\n",
    "            Crash=True\n",
    "        if Is_arrive(Next_x,Next_y,im)==True:\n",
    "            Arrive=True\n",
    "    elif Action==1:\n",
    "        Next_x=Current_x\n",
    "        Next_y=Current_y\n",
    "        Next_angle=Current_angle-45\n",
    "    elif Action==2:\n",
    "        Next_x=Current_x\n",
    "        Next_y=Current_y\n",
    "        Next_angle=Current_angle+45        \n",
    "    True_angle,Left_right,Distance=Robot_destination(Destination_x,Destination_y,Next_x,Next_y,Next_angle)\n",
    "    State_number,Distance_level,Angle_level=Output_state_index(True_angle,Left_right,Distance)\n",
    "    if Arrive==False:\n",
    "        Reward=Distance_level*-0.5-Angle_level\n",
    "    elif Arrive==True:\n",
    "        Reward=50\n",
    "    return Next_x,Next_y,Next_angle,Reward,State_number,Arrive,Crash\n",
    "#Next_state_F2(600,600,50,1800,1800,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Movement_plot(Vec_x,Vec_y,Destination_x,Destination_y):\n",
    "    im=Image.new(\"RGB\", size=(1000,1000),color=(0,0,0))\n",
    "    draw=ImageDraw.Draw(im,mode='RGB')\n",
    "    draw.rectangle((40,40,960,960),255,255)\n",
    "    draw.ellipse((Destination_x-20,Destination_y-20,Destination_x+20,Destination_y+20),(255,0,0),(255,0,0))\n",
    "    for i,j in zip(Vec_x,Vec_y):\n",
    "        draw.ellipse((i-40,j-40, i+40,j+40),150,150) \n",
    "    return im\n",
    "\n",
    "# plt.imshow(im)\n",
    "# plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "***********************************************************\n",
      "Succeed to initialize Q-table2!\n",
      "***********************************************************\n"
     ]
    }
   ],
   "source": [
    "Q_Table2=Initial_Q_Table2(Q_table2_actions_length,Q_table2_states_length)\n",
    "global Epoche\n",
    "Epoche=0\n",
    "global Action_times #Rigister the totality of the times of selecting actions, including the random selections and selection based on Q_Table\n",
    "Action_times=0\n",
    "global Epoche_action_interval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "Q_table2=np.loadtxt(r\"E:\\Graduate\\python\\Q_Table_notgait_finish\\Q_Table2_notgait\\Q_Table2_notgait_2.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAARUAAAD8CAYAAABZ0jAcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEBNJREFUeJzt3X+s3Xddx/Hny5YNGL86SZrS1qwmBVP+0EEzNyCEMHAT\nCd1fS0kWi077h9MA/gGt/GH8bxpC0BiIDT+sgmuasbhmEWEMEvUPNgpTWduVFerW1m6dWQRDzGDj\n7R/nWzl0be+953zuOd9zzvPRNPd7vvd77nmf9u65749zelNVSFIrPzftASTNF6MiqSmjIqkpoyKp\nKaMiqSmjIqmpiUclyc1Jjic5kWTPpB9f0urKJF+nkmQN8B3gncBp4BvAe6vq6MSGkLSqJr2nch1w\noqq+V1U/Ag4AOyY8g6RVtHbCj7cRODV0+zTwqxdulGQ3sLu7+cYJzCUttKpKq6816agsS1XtA/YB\nJPF9BNIMmfThzxlg89DtTd06SXNi0lH5BrA1yZYkVwA7gUMTnkHSKpro4U9VPZfk94EvAWuAz1TV\nkUnOIGl1TfSS8ig8pyKtvpYnan1FraSmjIqkpoyKpKZ6+TqVler7eSFpmpJmp0uWxT0VSU0ZFUlN\nGRVJTRkVSU0ZFUlNGRVJTRkVSU0ZFUlNGRVJTRkVSU0ZFUlNGRVJTRkVSU0ZFUlNGRVJTRkVSU0Z\nFUlNGRVJTRkVSU0ZFUlNGRVJTRkVSU0ZFUlNGRVJTRkVSU0ZFUlNGRVJTRkVSU0ZFUlNrZ32APMi\n5P+Xi5riJNJ0uafSwHBQzt++cJ20KEaOSpLNSb6W5GiSI0ne362/Osn9SR7rPq4bus/eJCeSHE9y\nU4snMG2Xi4dh0SJK1Wi76kk2ABuq6ltJXg58E7gFeB/wTFXdmWQPsK6qPpxkG3AXcB3wGuArwGur\n6vklHmfJAUd9DuNabjQ8HNI0JUt/n1ZVs/8DjrynUlVnq+pb3fL/AMeAjcAOYH+32X4GoaFbf6Cq\nnq2qk8AJBoGZe+6xaJE0OaeS5BrgWuBBYH1Vne0+9SSwvlveCJwautvpbt3Fvt7uJIeTHG4xXx8Y\nFi2KsaOS5GXAF4APVNUPhj9Xg+OSFe/7V9W+qtpeVdvHnW81rfSwxrBoEYwVlSQvYhCUz1fVPd3q\np7rzLefPu5zr1p8BNg/dfVO3bqEYFs27ca7+BPg0cKyqPjb0qUPArm55F3Dv0PqdSa5MsgXYCjw0\n6uP3xSgnYQ2L5tk4V3/eAvwz8G3gJ93qP2JwXuUg8AvA48CtVfVMd5+PAL8NPMfgcOmLy3ic3l79\nGbbSUHhFSJMy6as/I0dlUmYlKjDaHohx0WqbmUvKeiEDIRmV5ka5IuQ5Fs0To7IKPHmrRWZUVolh\n0aIyKquoul8rYVg064xKDxkWzTKjMgHusWiRGBVJTRmVCfJysxaBUZkwrwpp3hmVKTAsmmdGZUoM\ni+aVUZkiw6J5ZFSmzMvNmjdGZUYZFvWVUekJD4U0L4xKjxgWzQOj0jOGRbPOqPSQYdEsMyo9ZVg0\nq4xKjxkWzSKj0nOGRbPGqMwAw6JZYlRmhGHRrDAqM8SwaBYYlRljWNR3RmUGGRb1mVGZUYZFfWVU\nZphhUR8ZlRlnWNQ3RmUOGBb1iVGZE4ZFfWFU5ohhUR8YlTljWDRtY0clyZokDye5r7t9dZL7kzzW\nfVw3tO3eJCeSHE9y07iPrYszLJqmFnsq7weODd3eAzxQVVuBB7rbJNkG7AReD9wMfCLJmgaPr4sw\nLJqWsaKSZBPwG8CnhlbvAPZ3y/uBW4bWH6iqZ6vqJHACuG6cx9flGRZNw7h7Kh8HPgT8ZGjd+qo6\n2y0/CazvljcCp4a2O92te4Eku5McTnJ4zPkWnmHRpI0clSTvBs5V1TcvtU1VFaz8u7qq9lXV9qra\nPup8+inDoklaO8Z93wy8J8m7gBcDr0jyOeCpJBuq6mySDcC5bvszwOah+2/q1mkCijIUmoiR91Sq\nam9VbaqqaxicgP1qVd0GHAJ2dZvtAu7tlg8BO5NcmWQLsBV4aOTJtWKj/HhVQ6SVGmdP5VLuBA4m\nuR14HLgVoKqOJDkIHAWeA+6oqudX4fF1Ge6xaLVlcNqjv5IsOWDfn0MfjRKWUc7NaPqSpf+uq6rZ\n/2l8Re2C8uStVotRWWCGRavBqCy46n6thGHR5RgVjcSw6FKMigD3WNSOUZHUlFHRz/AFchqXUdEL\neFVI4zAquijDolEZFV2SYdEojIouy7BopYyKluTlZq2EUdGqMSyLyaho2TwU0nIYFa2IYdFSjIpW\nzLDocoyKRmJYdClGRSMzLLoYo6KxGBZdyKhobIZFw4yKmjAsOs+oqBnDIjAqasywyKioOcOy2IyK\nVoVhWVxGRavGsCwmo6JVZVgWj1HRqjMsi8WoaCIMy+IwKpoYw7IYjIomapSwaLYYFU2cP7BsvhkV\nTYWHQvPLqGhqDMt8MiqaKsMyf8aKSpJXJbk7yaNJjiW5IcnVSe5P8lj3cd3Q9nuTnEhyPMlN44+v\neWBY5su4eyp/DvxjVf0S8MvAMWAP8EBVbQUe6G6TZBuwE3g9cDPwiSRrxnx8zQl/YNn8GDkqSV4J\nvBX4NEBV/aiq/hvYAezvNtsP3NIt7wAOVNWzVXUSOAFcN+rjS+qncfZUtgBPA59N8nCSTyW5Clhf\nVWe7bZ4E1nfLG4FTQ/c/3a17gSS7kxxOcniM+TSDvNw8+8aJylrgDcAnq+pa4Id0hzrnVVXByg+Y\nq2pfVW2vqu1jzKcZ5QvkZts4UTkNnK6qB7vbdzOIzFNJNgB0H891nz8DbB66/6ZunfQC7rHMrpGj\nUlVPAqeSvK5bdSNwFDgE7OrW7QLu7ZYPATuTXJlkC7AVeGjUx9f886rQbFo75v3/APh8kiuA7wG/\nxSBUB5PcDjwO3ApQVUeSHGQQnueAO6rq+TEfX3OuqBWHIsRDqCnK4LRHfyVZcsC+Pwe1sdK4GJaB\nZOk/t6pqtovnK2o1tzwUmg6jopnhC+Rmg1GR1JRR0czxcnO/GRXNJC8395dR0cwyLP1kVDTTDEv/\nGBXNPMPSL0ZFc8HLzf1hVLTQDEt7RkVzxUOh6TMqmjuGZbqMiuaSYZkeo6K5ZVimw6horhmWyTMq\nmnuGZbKMihaCYZkco6KFYVgmw6hooRiW1WdUtHAMy+oyKlpIhmX1GBUtLMOyOoyKFpphac+oaOEZ\nlraMioRhacmoSB3D0oZRkYYYlvEZFekC/gzm8RgV6SL8gWWjMyrSJbjHMhqjIl2G/0L/yhkVaQnu\nsayMUZGWwbAs39ppDyDNivNhudwhTmWwxc+uXKwgjbWnkuSDSY4keSTJXUlenOTqJPcneaz7uG5o\n+71JTiQ5nuSm8ceXZkAW6zxLasSKJtkI/Auwrar+N8lB4B+AbcAzVXVnkj3Auqr6cJJtwF3AdcBr\ngK8Ar62q55d4nCUHHPU5SOMY3mOp5XRjSt+nWUbUqpb1DJZl3HMqa4GXJFkLvBT4T2AHsL/7/H7g\nlm55B3Cgqp6tqpPACQaBkWZSDf3ST40clao6A3wUeAI4C3y/qr4MrK+qs91mTwLru+WNwKmhL3G6\nW/cCSXYnOZzk8KjzSZqOkaPSnSvZAWxhcDhzVZLbhrepwXHJijNeVfuqantVbR91PknTMc7hzzuA\nk1X1dFX9GLgHeBPwVJINAN3Hc932Z4DNQ/ff1K2TNEfGicoTwPVJXprBmaAbgWPAIWBXt80u4N5u\n+RCwM8mVSbYAW4GHxnh8qT+WOgm7QBcTRn6dSlU9mORu4FvAc8DDwD7gZcDBJLcDjwO3dtsf6a4Q\nHe22v2OpKz/STKm6+OXjBQoKjHFJeVK8pCyNZ9YuKUvSzzAqkpoyKpKaMiqSmjIqkpoyKpKaMiqS\nmjIqkpoyKpKaMiqSmjIqkpoyKpKaMiqSmjIqkpoyKpKaMiqSmjIqkpoyKpKaMiqSmjIqkpoyKpKa\nMiqSmjIqkpoyKpKaMiqSmjIqkpoyKpKaMiqSmjIqkppaO+0BWljOT7WXNBnuqUhqyqhIasqoSGrK\nqEhqyqhIamrJqCT5TJJzSR4ZWnd1kvuTPNZ9XDf0ub1JTiQ5nuSmofVvTPLt7nN/ES/ZSHNpOXsq\nfw3cfMG6PcADVbUVeKC7TZJtwE7g9d19PpFkTXefTwK/C2ztfl/4NSXNgSWjUlX/BDxzweodwP5u\neT9wy9D6A1X1bFWdBE4A1yXZALyiqr5eVQX8zdB9JM2RUV/8tr6qznbLTwLru+WNwNeHtjvdrftx\nt3zh+otKshvY3d18FnjkUtv2zKuB/5r2ECswS/PO0qwwW/O+ruUXG/sVtVVVSarFMENfcx+wDyDJ\n4ara3vLrr5ZZmhVma95ZmhVma94kh1t+vVGv/jzVHdLQfTzXrT8DbB7ablO37ky3fOF6SXNm1Kgc\nAnZ1y7uAe4fW70xyZZItDE7IPtQdKv0gyfXdVZ/fHLqPpDmy5OFPkruAtwGvTnIa+GPgTuBgktuB\nx4FbAarqSJKDwFHgOeCOqnq++1K/x+BK0kuAL3a/l2Pfcp9MD8zSrDBb887SrDBb8zadNYOLMZLU\nhq+oldSUUZHUVG+jkuTm7qX+J5Ls6cE8m5N8LcnRJEeSvL9bv+K3LEx47jVJHk5yX5/nTfKqJHcn\neTTJsSQ39HXW7vE/2H0fPJLkriQv7tO8U317TVX17jewBvgu8IvAFcC/AdumPNMG4A3d8suB7wDb\ngD8D9nTr9wB/2i1v6+a+EtjSPZ81U5j7D4G/A+7rbvdyXgavzP6dbvkK4FU9nnUjcBJ4SXf7IPC+\nPs0LvBV4A/DI0LoVzwc8BFwPhMHFlV9f8rEn/U2+zD+QG4AvDd3eC+yd9lwXzHgv8E7gOLChW7cB\nOH6xmYEvATdMeMZNDN6b9fahqPRuXuCV3X+kuWB972btHm8jcAq4msEV1PuAX+vbvMA1F0RlRfN1\n2zw6tP69wF8t9bh9Pfw5/5d23mVf1j9pSa4BrgUe5PJvWZj2c/g48CHgJ0Pr+jjvFuBp4LPdodqn\nklzV01mpqjPAR4EngLPA96vqy/R03iErnW8jK3h7zXl9jUpvJXkZ8AXgA1X1g+HP1SDnvbhGn+Td\nwLmq+ualtunRvGsZ7Kp/sqquBX5I987383o0K925iB0MYvga4Koktw1v06d5L2Y15+trVC71cv+p\nSvIiBkH5fFXd061e6VsWJuXNwHuS/AdwAHh7ks/Rz3lPA6er6sHu9t0MItPHWQHeAZysqqer6sfA\nPcCbejzveRN5e01fo/INYGuSLUmuYPBvtBya5kDdWe9PA8eq6mNDn1rRWxYmNW9V7a2qTVV1DYM/\nv69W1W19nLeqngROJTn/btkbGbwqu3ezdp4Ark/y0u774kbgWI/nPW8yb6+Z1MmtEU4yvYvBFZbv\nAh/pwTxvYbC7+O/Av3a/3wX8PIOToY8BXwGuHrrPR7r5j7OMs+arOPvb+OmJ2l7OC/wKcLj78/17\nYF1fZ+0e/0+ARxn8sxx/y+DKSW/mBe5icL7n/D87cvso8wHbu+f4XeAvueBk+sV++zJ9SU319fBH\n0owyKpKaMiqSmjIqkpoyKpKaMiqSmjIqkpr6PxM0U5SDOREzAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1c248ca8e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib qt5\n",
    "def Draw_result(Destination_x,Destination_y,Start_x,Start_y,Start_angle,Q_table):\n",
    "    Path_x=[]\n",
    "    Path_y=[]\n",
    "    Current_x=Start_x\n",
    "    Current_y=Start_y\n",
    "    Current_angle=Start_angle\n",
    "    Arrive=False\n",
    "    im=Draw_map(Destination_x,Destination_y)\n",
    "    while Arrive==False:\n",
    "        Path_x.append(Current_x)\n",
    "        Path_y.append(Current_y)\n",
    "        True_angle,Left_right,Distance=Robot_destination(Destination_x,Destination_y,\n",
    "                                                             Current_x,Current_y,Current_angle)\n",
    "        Current_state_number,Distance_level,Angle_level=Output_state_index(True_angle,\n",
    "                                                                               Left_right,Distance)\n",
    "        State_action=Q_table2[Current_state_number,:]\n",
    "        Next_action=np.argmax(State_action)\n",
    "        Next_x,Next_y,Next_angle,Reward,Next_state_number,Arrive,Crash=Output_next_state(Current_x,Current_y,\n",
    "                                                                    Current_angle,Destination_x,Destination_y,Next_action,im)\n",
    "        Current_x=Next_x\n",
    "        Current_y=Next_y\n",
    "        Current_angle=Next_angle\n",
    "    im_result=Image.new(\"RGB\", size=(1000,1000),color=(0,0,0))\n",
    "    draw_result=ImageDraw.Draw(im_result,mode='RGB')\n",
    "    draw_result.rectangle((40,40,960,960),(255,255,255),(255,255,255))\n",
    "    draw_result.ellipse((Destination_x-20,Destination_y-20,Destination_x+20,Destination_y+20),(255,0,0),(255,0,0))\n",
    "    print(len(Path_x))\n",
    "    for d in range(36):\n",
    "        for i,j in zip(Path_x[2*d:2*d+2],Path_y[2*d:2*d+2]):\n",
    "            draw_result.ellipse((i-20,j-20, i+20,j+20),(0,255,0),(0,255,0))\n",
    "            plt.imshow(im_result)\n",
    "            plt.savefig((r\"E:\\Graduate\\python\\Q_Table_notgait_finish\\Q_Table2_notgait\\Pic\\picture\")+str(d)+\".png\")\n",
    "    return im_result\n",
    "im_result=Draw_result(800,800,200,200,0,Q_Table2)\n",
    "plt.imshow(im_result)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.savetxt(\"Q_Table2_notgait2.txt\",Q_Table2_final)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
